#!/usr/bin/env expect
############################################################################
# Purpose: Test of NRT functionality
#          Test of switch/nrt options (--network option).
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2012 SchedMD LLC
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set test_id    "13.2"
set exit_code  0
set test_code  0
set fail       0
set smd_system 1

print_header $test_id

if {[test_switch_nrt] == 0} {
	send_user "\nWARNING: This test is only compatible with switch/nrt systems\n"
	exit 0
}

#
# This tests a few options that should and should not work for NRT
#
proc create_nrtparams {nrt_params} {

	global bin_id srun fail

	set fail 0
	eval spawn $srun -N1-2 -n2 -O --network=$nrt_params $bin_id
	expect {
		-re "error" {
			set fail 1
			exp_continue
		}
		timeout {
			send_user "\nFAILURE: srun not responding\n"
			set exit_code 1
			exp_continue
		}
		eof {
			wait
		}
	}

}

proc create_nrtparams_4tasks {nrt_params} {

	global bin_id srun fail

	set fail 0
	eval spawn $srun -N1-2 -n4 -O --network=$nrt_params $bin_id 
	expect {
		-re "error" {
			set fail 1
			exp_continue
		}
		timeout {
			send_user "\nFAILURE: srun not responding\n"
			set exit_code 1
			exp_continue
		}
		eof {
			wait
		}
	}
}


#
# This makes a list of all the bad test to be tested.
#
send_user "\n===============Bad Tests===================\n"
if {$smd_system == 1} {
#	For use on IB system
	set badtest "
	{sn_all,devtype=hfi}
	{sn_single,devtype=hfi}
	{sn_single,devtype=iponly,us}
	"
} else {
#	For use on HFI/Torrent system
	set badtest "
	{sn_all,devtype=ib}
	{sn_single,devtype=ib}
	{sn_single,devtype=iponly,us}
	"
}

foreach test $badtest {
	set test_code [create_nrtparams $test]
	if {$fail == 1} {
		send_user "Error expected here, don't worry :)\n"
		send_user "fail_code = $fail\n\n"
	} else {
		send_user "FAILURE: This test should have failed but did not.\n"
		send_user "fail_code = $fail\n\n"
		set exit_code  1
	}

}

#
# This test a list of all the good tests that should succeed.
#
send_user "\n===============Good Tests================\n"
if {$smd_system == 1} {
#	For use on IB system
	set goodtest "
	{sn_all,devtype=ib}
	{sn_all,devtype=iponly}
	{sn_single,devtype=ib}
	{sn_single,devtype=ib,us}
	{sn_single,devtype=iponly,ipv4}
	{sn_single,devtype=iponly,ipv6}
	{sn_all,ipv6}
	{devname=mlx4_0}
	{devname=eth0}
	{devname=mlx4_0,mpi,lapi}
	{devname=mlx4_0,mpi,lapi,us}
	{devname=mlx4_0,mpi,lapi,instances=3}
	{sn_all}
	{sn_all,mpi,lapi}
	{sn_all,mpi,lapi}
	{devname=mlx4_0,bulk_xfer,us}
	"
} else {
#	For use on HFI/Torrent system
	set goodtest "
	{sn_all,devtype=hfi}
	{sn_all,devtype=iponly}
	{sn_single,devtype=hfi}
	{sn_single,devtype=hfi,us}
	{sn_single,devtype=iponly,ipv4}
	{sn_single,devtype=iponly,ipv6}
	{sn_all,ipv6}
	{devname=hfi0}
	{devname=eth0}
	{devname=hfi0,mpi,lapi}
	{devname=hfi0,mpi,lapi,us}
	{devname=hfi0,mpi,lapi,instances=3}
	{sn_all}
	{sn_all,mpi,lapi}
	{sn_all,mpi,lapi}
	{devname=hfi0,bulk_xfer,us}
	"
}

foreach test $goodtest {
	set test_code [create_nrtparams $test]
	if {$fail == 0} {
		send_user "This test should work :)\n"
		send_user "fail_code = $fail\n\n"
	} else {
		send_user "FAILURE: This test should not have failed\n"
		send_user "fail_code = $fail\n\n"
		set exit_code 1
	}
}

#
# This test a list of all the good test with 4 tasks
#
send_user "\n============Good Tests With 4 Tasks============\n"
if {$smd_system == 1} {
#	For use on IB system
	set good4task "
	{devname=mlx4_0,mpi,lapi,us,instances=3}
	{sn_all,mpi,lapi,instances=2}
	{sn_all,mpi,lapi,instances=3,us}
	"
} else {
#	For use on HFI/Torrent system
	set good4task "
	{devname=hfi0,mpi,lapi,us,instances=3}
	{sn_all,mpi,lapi,instances=2}
	{sn_all,mpi,lapi,instances=3,us}
	"
}

foreach test $good4task {
	set test_code [create_nrtparams_4tasks $test]
	if {$fail == 0} {
		send_user "This test should work :)\n"
		send_user "fail_code = $fail\n\n"
	} else {
		send_user "FAILURE: This test should not have failed\n"
		send_user "fail_code = $fail\n\n"
		set exit_code 1
	}
}

if {$exit_code != 0} {
	send_user "\nFAILURE: This test will only run on the SMD cluster without changes\n"
} else {
	send_user "\nSUCCESS\n"
}
exit $exit_code
